#!/usr/bin/perl
#!/cygdrive/c/cygwin/bin/perl.exe

use strict;
use warnings;
no strict "refs";
use threads;
use threads::shared;  
use Thread::Semaphore; 
use LWP::UserAgent;
use Fcntl qw(:flock);
use POSIX ":sys_wait_h";

my $pidsdir ="./pids";
my $resultfile = "products.xml";
my $urlfile = "urls.txt";
my $prjsemfile = "prjsem.tmp";
my $logfile = "log.txt";
my $logsemfile = "logsem.tmp";

my $RESULTHND;
my $URLHND;
my $LOGHND;

my $screenlog = 1;
my $picdir = "./pictures";
my $get_picture_flag = 0;

my %searchsites	= (
	'www.thefancy.com' => [
		'http://www.thefancy.com/search?q=iphone&pg=1',
		'\s+<a\shref="(/things/\d+/([^"]+))"',
		'thefancy_project',
		'<div\sclass="search-result\sempty">',
		'thefancy',
		'1',
		3, 15
		],
	'thegadgetflow.com' => [
		'http://thegadgetflow.com/page/1/',
		'<a\sclass="ajaxlink\sentry-title"\shref="(.+)">(.+)</a>',
		'thegadgetflow_project',
		'404_go.png"\s/></a>',
		'thegadgetflow',
		'0',
		2, 5
		],
	'www.kickstarter.com' => [
		'http://www.kickstarter.com/projects/search?page=1&term=iphone',	#searchstr
		'"(.+\\?ref=search)">([^<>]+)<\\/a>',															#linkstr
		'kickstarter_project',																						#funcstr
		'<span>Sorry\,\sno\sresults\sfor',																#finalstr
		'kickstarter',																										#typestr
		'1',																															#keyflag
		3, 15                                                             #threads number for targets, threads number for project
		],
	'www.indiegogo.com' => [
		'http://www.indiegogo.com/projects?filter_title=iphone&pg_num=1&search_submit=Search&utf8=%E2%9C%93',
		'"(.+)"\sclass="name bold">(.+)</a>',
		'indiegogo_project',
		'<b>No\sresults\sfound\.',
		'indiegogo',
		'1',
		3, 15
		]
	);
	
my @searchkeywords = (
	"iphone",
	"ipad",
	);

# (hostname, categorystr, searchstr, linkstr, funcstr, finalstr, typestr, keyflag, thrnum1, thrnum2), (...), ...
my @searchtargets;

my $hostnameidx = 0;
my $categoryidx = 1;
my $searchidx = 2;
my $linkidx = 3;
my $funcidx = 4;
my $finalidx = 5;
my $typeidx = 6;
my $keyflagidx = 7;
my $thrnum1 = 8; #for target
my $thrnum2 = 9; #for project

my %oldurls						: shared;

my @project_urls			: shared;
my @project_names			: shared;
my @project_categorys	: shared;

my $target_num				: shared;
my @target_next_flags : shared;

my $project_urls_sem = Thread::Semaphore->new(0);
my $project_urls_mutex = Thread::Semaphore->new(1);
my $next_flags_mutex = Thread::Semaphore->new(1);

my $TARGET_THREADS_NUM = 5;
my $PROJECT_THREADS_NUM = 30;
my $RETRIVE_PAGE_RETRY_TIMES = 3;
my $MAX_SEARCH_PAGES = 300;

sub binarySearch {
  my ($list, $query) = @_;
  my ($low, $high)   = ($[, $#$list);

  while ( $low <= $high ) {
		my $try = int( ($low + $high) / 2 );

		# note: "lt" and "gt" because we are comparing strings
		# if we were comparing numbers then we would use "<" and ">"
		$list->[$try] lt $query and do { $low  = $try + 1; next };
		$list->[$try] gt $query and do { $high = $try - 1; next };

		return $try; # By this point it must be equal!
  }
  
  return -1;
}

sub is_url_exist
{
	my $ret;
	my $SEM;
	my $url = $_[0];

	$SEM = get_lock($prjsemfile);
	$ret = defined($oldurls{$url});
	release_lock($SEM);
	
	return $ret;
}

sub get_lock
{
	my $file = $_[0];
	my $SEM;

	open($SEM, ">$file") || die "cannot create semaphore: $!\n";
	flock($SEM, LOCK_EX) || die "lock failed\n";
	
	return $SEM;
}

sub release_lock
{
	my $SEM = $_[0];
	
	close($SEM);
}

sub mylog_f
{
	my $SEM = get_lock($logsemfile);
	
	# avoid die with lock
	eval
	{
		if ($screenlog)
		{
			printf $LOGHND @_;
			printf STDERR @_;
		}
		else
		{
			printf $LOGHND @_;
		}

		{
			my $old_fh = select($LOGHND);
			$|++;
			select($old_fh);
		}
	};
	
	if ($@){}

	release_lock($SEM);
}

sub mylog
{
	my $SEM = get_lock($logsemfile);

	# avoid die with lock
	eval
	{
		if ($screenlog)
		{
			print $LOGHND @_;
			print STDERR @_;
		}
		else
		{
			print $LOGHND @_;
		}

		{
			my $old_fh = select($LOGHND);
			$|++;
			select($old_fh);
		}
	};
	
	if ($@){}

	release_lock($SEM);
}

sub retrieve_page_really
{
	my $url = $_[0];
	my $host = $_[1];
	my $file = $_[2];
	my $check404 = $_[3];
	my $OUT;
	my $evalret;
	
	$evalret = eval
	{
		my $ua;
		
		if ($url =~ /^https/)
		{
			$ua = LWP::UserAgent->new(ssl_opts => { verify_hostname => 1 });
		}
		else
		{
			$ua = LWP::UserAgent->new;
		}
		
		$ua->timeout(10);
		#$ua->cookie_jar({}); 
		$ua->protocols_allowed(['http','https']);
		
		my %Chrome = (
			'Accept'=>'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
			'Host'=>$host,
			#'Connection'=>'keep-alive',
			#'Cache-Control'=>'max-age=0',
			#'Accept-Encoding'=>'gzip,deflate,sdch',
			#'Accept-Language'=>'zh-CN,zh;q=0.8',
			#'Accept-Charset'=>'GBK,utf-8;q=0.7,*;q=0.3',
			'User-Agent'=>'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22'
			);
		
		my $h = HTTP::Headers->new(%Chrome);
		my $req = HTTP::Request->new('GET', $url, $h);
		my $response = $ua->request($req);
	
		if (!$response->is_success)
		{
			my $code;
			
			$code = $response->code();
			
			if ($code != 404)
			{
				#mylog("retrieve_page Error($host,$url): ".$response->status_line."\n");
				return -1;
			}
			
			if ($check404)
			{
				return -1;
			}
		}
		
		if (!open($OUT, ">$file"))
		{
			mylog("open $file failed\n");
			return -1;
		}
		
		print $OUT $response->content; # xxx: really need to save contenct to a file???
		close($OUT);
		
		return 0;
	};
	
	if ($@)
	{
		#unlink($file);
		#mylog("retrieve_page: catch exception, $@\n");
		return -1;
	}
	#mylog("retrieve_page: $url successfully"."\n");
	
	return $evalret;
}

sub retrieve_page
{
	my $url = $_[0];
	my $host = $_[1];
	my $file = $_[2];
	my $retry_times = 0;
	
	while ($retry_times < $RETRIVE_PAGE_RETRY_TIMES)
	{
		if (retrieve_page_really($url, $host, $file, 0) >= 0)
		{
			return 0;
		}

		$retry_times++;
		
		mylog("retrieve_page $url failed, retry...\n");
	}
	
	mylog("give up retrieve_page $url\n");
	
	return -1;
}

sub retrieve_picurl
{
	my $url = $_[0];
	my $host = $_[1];
	my $file = $_[2];
	my $retry_times = 0;
	
	while ($retry_times < $RETRIVE_PAGE_RETRY_TIMES)
	{
		if (retrieve_page_really($url, $host, $file, 1) >= 0)
		{
			return 0;
		}

		$retry_times++;
		
		mylog("retrieve_page $url failed, retry...\n");
	}
	
	mylog("give up retrieve_page $url\n");
	
	return -1;
}

sub xmlprocess
{
	my($origin) = $_[0];
	my $double_quot = "\""; #xxx: keep ultraedit highlight right
	my $single_quote = "\'";

	#& - &amp;
	#< - &lt;
	#> - &gt;
	#" - &quot;
	#' - &#39;
	
	$origin =~ s/&/&amp;/g;
	$origin =~ s/</&lt;/g;
	$origin =~ s/>/&gt;/g;
	$origin =~ s/$double_quot/&quot;/g;
	$origin =~ s/$single_quote/&#39;/g;
	
	return $origin;
}

sub print_project
{
	my $picture;
	my($createtime, $name, $projecttype, $category, $description, $orderlink, $projectlink, $videolink,$widgetlink, $price, $pictures) = @_;

	my $SEM = get_lock($prjsemfile);
	
	if (defined($oldurls{$projectlink}))
	{
		release_lock($SEM);
		return;
	}
	
	$oldurls{$projectlink} = 1;

	# avoid die with lock
	eval
	{		
		print $RESULTHND "<Product>\n";
		print $RESULTHND "<CreateTime>".xmlprocess($createtime)."</CreateTime>\n";
		print $RESULTHND "<Name>".xmlprocess($name)."</Name>\n";
		print $RESULTHND "<ProjectType>".xmlprocess($projecttype)."</ProjectType>\n";
		print $RESULTHND "<Category>".xmlprocess($category)."</Category>\n";
		print $RESULTHND "<Description>\n".xmlprocess($description)."\n</Description>\n";
		print $RESULTHND "<OrderLink>".xmlprocess($orderlink)."</OrderLink>\n";
		print $RESULTHND "<ProjectLink>".xmlprocess($projectlink)."</ProjectLink>\n";
		print $RESULTHND "<VideoLink>".xmlprocess($videolink)."</VideoLink>\n";
		print $RESULTHND "<WidgetLink>".xmlprocess($widgetlink)."</WidgetLink>\n";
		print $RESULTHND "<Price>".xmlprocess($price)."</Price>\n";
		
		print $RESULTHND "<Pictures>\n";
		
		while ($picture = shift @{$pictures})
		{
			print $RESULTHND "<Picture><Src>".xmlprocess($picture)."</Src></Picture>\n";
		}
		
		print $RESULTHND "</Pictures>\n";
		
		print $RESULTHND "</Product>\n";

		{
			my $old_fh = select($RESULTHND);
			$|++;
			select($old_fh);
		}

		#xxx: remember it
		print $URLHND "$projectlink\n";
	
		{
			my $old_fh = select($URLHND);
			$|++;
			select($old_fh);
		}
	};
	
	if ($@){}

	release_lock($SEM);
}

sub retrieve_pics
{
	my $pics = $_[0];
	my $host = $_[1];
	my $hostname;
	my $projecttype = $_[2];
	my $file;
	my $pic;
	my @newpics;

	while ($pic = shift @{$pics})
	{
		my $lastname;
		my $special = 0;

		if ($pic =~ /.+\/(.+)/)
		{
			$lastname = $1;
			
			if ($lastname =~ /(.+)\?(.+)/)
			{
				$lastname = "$2"."_"."$1";
				$special = 1;
			}
		}
		else
		{
			next;
		}

		if ($pic =~ /https?:\/\/([^\/]+)\/.+/)
		{
			$hostname = $1;
		}
		else
		{
			next;
		}
		
		$file = "$picdir/".$projecttype."_".$lastname;

		if ($special)
		{
			print "picture: $pic, $hostname, $file\n";
		}
		
		if (retrieve_picurl($pic, $hostname, $file) < 0)
		{
			mylog("retrieve_page $pic failed\n"); 
			next;
		}
		
		push(@newpics, $pic);
	}
	
	if (!@newpics)
	{
		return -1;
	}

	@{$pics} = @newpics;
	
	return 0;
}

sub indiegogo_project
{
	my $name = $_[0];
	my $host = $_[1];
	my $url = $_[2];
	my $file = $_[3];
	my $projecttype = $_[4];
	my $category = $_[5];
	my $line;
	my $pledge = 0;
	my $backers = -1;
	my $shortdesc;
	my $description;
	my $widgetLink;
	my $tmpdesc = "";
	my $pledge_level = 0;
	my $divlevel = 0;
	my $mp4link;
	my $imagelink; #xxx: no need???
	my $PROJECT;
	my $launched_time;
	my @pictures;

	#1.project name, 2 project price(pledge) 3, short description .4, pictures.5. link
	if (!open($PROJECT, "<$file"))
	{
		mylog("open $file failed\n");
		return -1;
	}	
	
	while ($line = <$PROJECT>)
	{
		if ($line =~ /<div\sclass="amount\sbold">(\D*[0-9,]+\D*)<\/div>/)
		{
			my $tmp_backers = 0;
			my $tmp_pledge = $1;

			while ($line = <$PROJECT>)
			{
				if ($line =~ /<p\sclass="fl\sclaimed\sbig-perk-button">(\d+)\sclaimed/)
				{
					$tmp_backers = $1;
					last;
				}
				else
				{
					next;
				}
			}

			if ($tmp_backers > $backers)
			{
				$backers = $tmp_backers;
				$pledge = $tmp_pledge;
			}
			
			next;
		}
		
		if (!defined($shortdesc))
		{
			#if ($line =~ /<p\sclass="notranslate">(.+)<\/p>/)
			#{
			#	$shortdesc = $1;
			#	next;
			#}
			# multi lines
			my $tmpstr = "";
			
			if ($line =~ /<p\sclass="notranslate">/)
			{
				while (!($line =~ /<\/p>/))
				{
					$tmpstr .= $line;
					
					if (!($line = <$PROJECT>))
					{
						last;
					}
				}
				
				$tmpstr .= $line;
			}
			
			if ($tmpstr =~ /<p\sclass="notranslate">(.+)<\/p>/s)
			{
				$shortdesc = $1;
			}
		}

		if (!defined($widgetLink))
		{			
			if ($line =~ /<meta\sproperty="og:url"\scontent="http:\/\/www.indiegogo.com\/projects\/(\d+)\/fblk"/)
			{
				$widgetLink = "http:\/\/www.indiegogo.com\/project\/".$1."\/widget" ;
				next;
			}
		}

		# main picture <img alt="20130328184421-capture_03" src="http://d2oadd98wnjs7n.cloudfront.net/medias/729433/primary_pictures/full/20130328184421-capture_03.jpg?1364521467" width="620">
		if ($line =~ /primary_pictures/)
		{
			if($line =~ /<img.*?src="(.*?)"/)
			{
				push(@pictures, $1);	
				next;
			}
		}
		
		if (!defined($description))
		{
			if ($divlevel < 1)
			{
				if ($line =~ /<div\sclass="user-content">/)
				{
					$tmpdesc = $tmpdesc.$line;
					$divlevel++;
				}
			}
			else
			{
				if ($line =~ /<img\ssrc="(.*?)"/)
				{
					push(@pictures, $1);	
				}
				
				$tmpdesc = $tmpdesc.$line;
				
				if ($line =~ /<div/)
				{
					$divlevel++;
				}

				if ($line =~ /<\/div>/)
				{
					$divlevel--;
				}
				
				if ($divlevel < 1)
				{
					$description = $tmpdesc;
				}
			}
		}

		if (!defined($mp4link))
		{
		    #<iframe src="http://player.vimeo.com/video/59640097?title=0&amp;portrait=0" width="620" height="465" frameborder="0"></iframe>
			if ($line =~ /<iframe\ssrc="http:\/\/player.vimeo.com\/video\/(\S+)"\s/)
			{
				$mp4link = "http:\/\/player.vimeo.com\/video/".$1;
			}
		}
		
		if (!defined($launched_time))
		{
			if ($line =~ /<img\salt="(\d\d\d\d\d\d\d\d).+"/)
			{
				$launched_time = $1;
			}
		}
	}
	
	if (!defined($pledge) || !defined($shortdesc))
	{
		mylog("---wrong project infomation $url");
		!defined($pledge) && mylog(" pledge is undefined");
		!defined($shortdesc) && mylog(" shortdesc is undefined");
		!defined($launched_time) && mylog(" launched_time is undefined");
		mylog("\n");
	}
	else
	{
		if ($get_picture_flag && (retrieve_pics(\@pictures, $host, $projecttype) < 0))
		{
			mylog("get pictures failed for $url\n");
			close($PROJECT);
			return;
		}
		
		!defined($pledge) && ($pledge = "");
		!defined($shortdesc) && ($shortdesc = "");
		!defined($mp4link) && ($mp4link = "");
		!defined($launched_time) && ($launched_time = "20130101");
		!defined($widgetLink) && ($widgetLink = "");
	
		if ($launched_time =~ /(\d\d\d\d)(\d\d)(\d\d)/)
		{
			$launched_time =~ s/(\d\d\d\d)(\d\d)(\d\d)/$1-$2-$3/;
		}

		print_project($launched_time, $name, $projecttype, $category, $shortdesc, $url, $url, $mp4link, $widgetLink, $pledge, \@pictures);
	}
	
	close($PROJECT);
}

sub kickstarter_project
{
	my $name = $_[0];
	my $host = $_[1];
	my $url = $_[2];
	my $file = $_[3];
	my $projecttype = $_[4];
	my $category = $_[5];
	my $line;
	my $pledge = 0;
	my $backers = -1;
	my $shortdesc;
	my $description;
	my $tmpdesc = "";
	my $divlevel = 0;
	my $mp4link;
	my $imagelink; #xxx: no need???
	my $PROJECT;
	my $launched_time;
	my $launched_time_level = 0;
	my $widgetlink;
	my @pictures;

	#1.project name, 2 project price(pledge) 3, short description .4, pictures.5. link
	if (!open($PROJECT, "<$file"))
	{
		mylog("open $file failed\n");
		return -1;
	}	
	
	while ($line = <$PROJECT>)
	{
		if ($line =~ /<h5>Pledge\s(\D*[0-9,]+\D*)\sor\smore<\/h5>/)
		{
			my $tmp_backers = 0;
			my $tmp_pledge = $1;

			while ($line = <$PROJECT>)
			{
				if ($line =~ /(\d+)\sbackers/)
				{
					$tmp_backers = $1;
					last;
				}
				else
				{
					next;
				}
			}
			
			if ($tmp_backers > $backers)
			{
				$backers = $tmp_backers;
				$pledge = $tmp_pledge;
			}
			
			next;
		}

		if (!defined($shortdesc))
		{
			if (
				($line =~ /<p\sclass="short-blurb\sbig_type">/) ||
				($line =~ /<p\sclass="big_type">/)
				)
			{
				$shortdesc = "";
				
				while ($line = <$PROJECT>)
				{
					if ($line =~ /<\/p>/)
					{
						last;
					}

					$shortdesc = "$shortdesc"."$line";
				}
				
				next;
			}
		}
		
		#<meta property="og:image" content="https://s3.amazonaws.com/ksr/projects/445759/photo-main.jpg?1359253234"/>
		if ($line =~ /<meta\sproperty="og:image"\scontent="(.*?)"/)
		{
			push(@pictures, $1);	
		}

		if (!defined($description))
		{
			if ($divlevel < 1)
			{
				if ($line =~ /<div\sclass="full-description">/)
				{
					$tmpdesc = $tmpdesc.$line;
					$divlevel++;
				}
			}
			else
			{
				if ($line =~ /<img\ssrc="(.*?)">/)
				{
					push(@pictures, $1);	
				}
				
				$tmpdesc = $tmpdesc.$line;
				
				if ($line =~ /<div/)
				{
					$divlevel++;
				}

				if ($line =~ /<\/div>/)
				{
					$divlevel--;
				}
				
				if ($divlevel < 1)
				{
					$description = $tmpdesc;
				}
			}
		}
		
		if (!defined($mp4link))
		{
			if ($line =~ /data-video="([^\"]+)"/)
			{
				$mp4link = $1;
			}
		}
		
		if (!defined($launched_time))
		{
			if ($launched_time_level < 1)
			{
				if ($line =~ /<b>Launched:<\/b>/)
				{
					$launched_time_level++;
				}
			}
			else
			{
				if ($line =~ /([a-zA-Z]+\s\d{1,2}\,\s\d\d\d\d)/)
				{
					$launched_time = $1;
				}
			}
		}
	}

	if (!defined($pledge) || !defined($shortdesc))
	{
		mylog("---wrong project infomation $url");
		!defined($pledge) && mylog(" pledge is undefined");
		!defined($shortdesc) && mylog(" shortdesc is undefined");
		!defined($launched_time) && mylog(" launched_time is undefined");
		mylog("\n");
	}
	else
	{
		if ($get_picture_flag && (retrieve_pics(\@pictures, $host, $projecttype) < 0))
		{
			mylog("get pictures failed for $url\n");
			close($PROJECT);
			return;
		}
		
		!defined($pledge) && ($pledge = "");
		!defined($shortdesc) && ($shortdesc = "");
		!defined($mp4link) && ($mp4link = "");
		!defined($launched_time) && ($launched_time = "Jan 1, 2013");

		if ($launched_time =~ /(\w\w\w)\s(\d+),\s(\d\d\d\d)/)
		{
			my %mons = (
				"Jan" => "01",
				"Feb" => "02",
				"Mar" => "03",
				"Apr" => "04",
				"May" => "05",
				"Jun" => "06",
				"Jul" => "07",
				"Aug" => "08",
				"Sep" => "09",
				"Oct" => "10",
				"Nov" => "11",
				"Dec" => "12",
				);
			my $mon = $mons{$1};
			my $day = $2;
			my $year = $3;
			
			if (!defined($mon))
			{
				mylog("invalid month value $mon\n");
				$mon = "Jan";
			}
			
			(length($day) < 2) && ($day = "0$day");
			(length($day) < 2) && ($day = "0$day");
			
			$launched_time = "$year-$mon-$day";
		}
		
		my $myurl = $url;
		if ($url =~ /(.*)\?ref=.*/)
		{
			$myurl = $1;
		}
		
		$widgetlink = $myurl."\/widget\/card.html";
        $mp4link = $myurl."\/widget\/video.html";
		
		print_project($launched_time, $name, $projecttype, $category, $shortdesc, $myurl, $myurl, $mp4link, $widgetlink, $pledge, \@pictures);
	}
	
	close($PROJECT);
	unlink($file);
}

sub thegadgetflow_project
{
	my $name = $_[0];
	my $host = $_[1];
	my $url = $_[2];
	my $file = $_[3];
	my $projecttype = $_[4];
	my $category = $_[5];
	my $line;
	my $pledge;
	#my $backers = -1;
	my $shortdesc;
	#my $description;
	#my $tmpdesc = "";
	#my $divlevel = 0;
	my $mp4link;
	my $imagelink; #xxx: no need???
	my $PROJECT;
	my $launched_time;
	my $widgetLink;
	my $orderLink;
	my @pictures;

	#1.project name, 2 project price(pledge) 3, short description .4, pictures.5. link
	if (!open($PROJECT, "<$file"))
	{
		mylog("open $file failed\n");
		return -1;
	}	
	
	while ($line = <$PROJECT>)
	{
		if (!defined($pledge))
		{
			if ($line =~ /<h2\sclass="entry-title">.+&#8211;\s(\D*[0-9,]+\D*)<\/h2>/)
			{
				$pledge = $1;
			}
			elsif ($line =~ /<h2\sclass="entry-title">.+-\s(\D*[0-9,]+\D*)<\/h2>/)
			{
				$pledge = $1;
			}
		}

		if (!defined($shortdesc))
		{
			if (($line =~ /<div\sclass="thecontent">/) || ($line =~ /<div\sclass="thecontent\sentry-content">/))
			{
				while(!defined($shortdesc) && $line)
				{
					if ($line =~ /<.*?>(.+)<\/.*?>/)
					{
						$shortdesc = $1;
						last;
					}
					
					$line = <$PROJECT>;
				}
			}
		}

		if ($line =~ /<div\sid="slidersmall"\sclass="slider">/)
		{
			while ($line = <$PROJECT>)
			{
				if ($line =~ /<div\sid="slidersmall-nav">/)
				{
					last;
				}
				else
				{
					if ($line =~ /<a\shref="(.*?)"/)
					{
						push(@pictures, $1);
						
						if (!defined($launched_time))
						{
							my $timestr = $1;
							
							if ($timestr =~ /uploads\/(\d\d\d\d)\/(\d\d)/)
							{
								$launched_time = "$1-$2-01";
							}
						}
					}
				}
			}
			
			next;
		}
		
		if (!defined($orderLink))
		{
			#1 <div class="button_sc"><a style="width: ;" href="http://fancy.to/af6hy1"><img class="thecart" src="http://cdn.thegadgetflow.com/wp-content/themes/thegadgetflow/images/cart.png">Buy Now</a></div>
			if ($line =~ /<div\sclass="button_sc.*?><a.*? href="(.+?)">.+<\/a>/)
			{
				$orderLink = $1;
			}
		}

	if ($line =~ /<div\sid="slider"\sclass="slider">/)
		{
			while ($line = <$PROJECT>)
			{
				if ($line =~ /<div\sid="slider-nav">/)
				{
					last;
				}
				else
				{
					if ($line =~ /<a\shref="(.*?)"/)
					{
						push(@pictures, $1);
						
						if (!defined($launched_time))
						{
							my $timestr = $1;
							
							if ($timestr =~ /uploads\/(\d\d\d\d)\/(\d\d)/)
							{
								$launched_time = "$1-$2-01";
							}
						}
					}
				}
			}
			
			next;
		}
		
		if (!defined($mp4link))
		{
			$mp4link = "";
		}
	}
	
	if (!defined($shortdesc))
	{
		mylog("---wrong project infomation $url");
		!defined($shortdesc) && mylog(" shortdesc is undefined");
		mylog("\n");
	}
	else
	{
		if ($get_picture_flag && (retrieve_pics(\@pictures, $host, $projecttype) < 0))
		{
			mylog("get pictures failed for $url\n");
			close($PROJECT);
			return;
		}
		
		!defined($pledge) && ($pledge = "$0");
		!defined($shortdesc) && ($shortdesc = "");
		!defined($mp4link) && ($mp4link = "");
		!defined($launched_time) && ($launched_time = "2013-01-01");
		!defined($mp4link) && ($mp4link = "");
		!defined($widgetLink) && ($widgetLink = "");
		
		my $myurl = $url;
		
		print_project($launched_time, $name, $projecttype, $category, $shortdesc, $orderLink, $myurl, $mp4link, $widgetLink, $pledge, \@pictures);
	}
	
	close($PROJECT);
	unlink($file);
}

sub thefancy_project
{
	my $name = $_[0];
	my $namewithoutdash;
	my $host = $_[1];
	my $url = $_[2];
	my $file = $_[3];
	my $projecttype = $_[4];
	my $category = $_[5];
	my $line;
	my $pledge;
	#my $backers = -1;
	my $shortdesc;
	#my $description;
	#my $tmpdesc = "";
	#my $divlevel = 0;
	my $mp4link;
	my $imagelink; #xxx: no need???
	my $PROJECT;
	my $launched_time;
	my $widgetLink;
	my @pictures;

	#1.project name, 2 project price(pledge) 3, short description .4, pictures.5. link
	if (!open($PROJECT, "<$file"))
	{
		mylog("open $file failed\n");
		return -1;
	}	
	
	while ($line = <$PROJECT>)
	{		
		if (!defined($pledge))
		{
			#if ($line =~ /<strong\sclass="price">(\D*[0-9,]+\D*)<\/strong>/)
			#{
			#	$pledge = $1;
			#}
			if ($line =~ /<meta\sproperty="og:title"\scontent="(.+)\s-\s(\D*[0-9,]+\D*)"><meta\sproperty="og:type"/)
			{
				if (defined($1) && !($1 eq ""))
				{
					$namewithoutdash = $1;
					$pledge = $2;
				}
			}
		}

		if (!defined($namewithoutdash))
		{
			if ($line =~ /<meta\sproperty="og:title"\scontent="(.+)"><meta\sproperty="og:type"/)
			{
				if (defined($1) && !($1 eq ""))
				{
					$namewithoutdash = $1;
				}
			}
		}
		
		if (!defined($shortdesc))
		{
			if ($line =~ /<div\sclass="description">(.+)<\/div>/)
			{
				$shortdesc = $1;
			}
		}

		if ($line =~ /<img\ssrc="(.+)" alt=""\swidth=\"/)
		{
			push(@pictures, $1);
		}

		if (!defined($launched_time))
		{
			if ($line =~ /<meta\sproperty="og:image"\scontent="http:\/\/.*?\/.*?\/(\d\d\d\d)(\d\d)(\d\d)\/.*?">/)
			{
				$launched_time = "$1-$2-$3";
			}
		}
		
		if (!defined($mp4link))
		{
			$mp4link = "";
		}
	}

	if (!defined($namewithoutdash))
	{
		$namewithoutdash = $name;
	}
	
	#if (!defined($pledge) || !defined($shortdesc))
	#{
	#	mylog("---wrong project infomation $url");
	#	!defined($pledge) && mylog(" pledge is undefined");
	#	!defined($shortdesc) && mylog(" shortdesc is undefined");
	#	mylog("\n");
	#}
	#else
	{
		if ($get_picture_flag && (retrieve_pics(\@pictures, $host, $projecttype) < 0))
		{
			mylog("get pictures failed for $url\n");
			close($PROJECT);
			return;
		}
		
		!defined($pledge) && ($pledge = "\$99");
		!defined($shortdesc) && ($shortdesc = "NO DESCRIPTION");
		!defined($mp4link) && ($mp4link = "");
		!defined($launched_time) && ($launched_time = "2013-01-01");
		!defined($mp4link) && ($mp4link = "");
		!defined($widgetLink) && ($widgetLink = "");
		
		my $myurl = $url;
		
		print_project($launched_time, $namewithoutdash, $projecttype, $category, $shortdesc, $myurl, $myurl, $mp4link, $widgetLink, $pledge, \@pictures);
	}
	
	close($PROJECT);
	unlink($file);
}

sub can_new_thread
{
	my ($mythreads, $max_thread_num) = @_;
	my $thread;
	my $mythreadnum;

	$mythreadnum = @{$mythreads};

	if ($mythreadnum >= $max_thread_num)
	{
		for (my $i = 0; $i < $mythreadnum; $i++)
		{
			$thread = shift @{$mythreads};
			
			if ($thread->is_joinable())
			{
				$thread->join();
			}
			else
			{
				if ($thread->is_running())
				{
					push(@{$mythreads}, $thread); # enqueue at tail again
				}
			}
		}
		
		$mythreadnum = @{$mythreads};
		
		if ($mythreadnum >= $mythreadnum)
		{
			return 0;
		}
	}
	
	return 1;
}

sub wait_child_thread
{
	my $thread;
	my ($mythreads) = @_;
	
	while ($thread = shift @{$mythreads})
	{
		if ($thread->is_joinable())
		{
			$thread->join();
		}
		else
		{
			if ($thread->is_running())
			{
				push(@{$mythreads}, $thread); # enqueue at tail again
				sleep(3);
			}
		}
	}
}

sub target_thread
{
	my ($target_url, $hostname, $tmpfile, $linkstr, $finalstr, $category, $mynum) = @_;

	if (retrieve_page($target_url, $hostname, $tmpfile) < 0)
	{
		mylog("retrieve_page $target_url failed\n"); 
		return;
	}

	my $line;
	my $TARGET;

	if (!open($TARGET, "<$tmpfile"))
	{
		mylog("open $hostname failed\n");
		return;
	}

	while($line = <$TARGET>)
	{
		if ($line =~ /$finalstr/)
		{
			$next_flags_mutex->down();
			$target_next_flags[$mynum] = 0;
			$next_flags_mutex->up();
			last;
		}

		if ($line =~ /$linkstr/)
		{
			my $project_url;
			
			if ($category eq "void")
			{
				$project_url = $1;
			}
			else
			{
				$project_url = "http://$hostname$1";

				if ($hostname eq "www.kickstarter.com")
				{
					if ($project_url =~ /(.*)\?ref=.*/)
					{
						$project_url = $1;
					}
				}
			}

			if (!is_url_exist($project_url))
			{
				$project_urls_mutex->down();
				push(@project_names, "$2");
				push(@project_urls, $project_url);
				push(@project_categorys, $category);
				$project_urls_mutex->up();
				
				$project_urls_sem->up();
			}
			else
			{
				mylog("existed $project_url\n");
			}
		}
	}
	
	close($TARGET);

	unlink($tmpfile);
}

sub target_main_thread
{
	my $mynum = $_[0];
	my $target_ref = $searchtargets[$mynum];
	my $hostname = $target_ref->[$hostnameidx];
	my $linkstr = $target_ref->[$linkidx];
	my $searchstr = $target_ref->[$searchidx];
	my $finalstr = $target_ref->[$finalidx];
	my $category = $target_ref->[$categoryidx];
	my $keyflag = $target_ref->[$keyflagidx];
	my @mythreads;
	my $thread;

	for (my $count=1; $count <= $MAX_SEARCH_PAGES; $count++)
	{
		my $target_url = $searchstr;
		my $tmpfile = $hostname."-$category"."-$count.tmp";

		if ($keyflag eq '0')
		{
			$target_url =~ s/\/\d\//\/$count\//;
		}
		else
		{
			$target_url =~ s/=\d/=$count/;
			$target_url =~ s/iphone/$category/;
		}

		mylog_f("target_main_thread($hostname): threads(%d)\n", $#mythreads + 1);

		while (!can_new_thread(\@mythreads, $TARGET_THREADS_NUM))
		{
			sleep(3);
		}
		
		$thread = threads->create(
			\&target_thread, $target_url, $hostname, $tmpfile, $linkstr, $finalstr, $category, $mynum
			);
		
		if (!defined($thread))
		{
			next;
		}
		
		push(@mythreads, $thread);

		$next_flags_mutex->down();
		
		if (!$target_next_flags[$mynum])
		{
			$next_flags_mutex->up();
			mylog("-------------$hostname($mynum): NEXT SEARCH ROUND-------------------\n");
			last;
		}
		
		$next_flags_mutex->up();
	}
	
	mylog("target_main_thread: wait_child_thread begin\n");
	wait_child_thread(\@mythreads);
	mylog("target_main_thread: wait_child_thread end\n");

	$project_urls_mutex->down();
	$target_num--;
	$project_urls_mutex->up();
	
	$project_urls_sem->up(); #xxx: if no urls is updated, need this up to inform project main thread
}

sub project_thread
{
	my $name = $_[0];
	my $hostname = $_[1];
	my $elem = $_[2];
	my $tmpfile = $_[3];
	my $category = $_[4];
	#xxx: index for searchsites
	my $target_ref = $searchsites{$hostname};
	my $prjfunc = @{$target_ref}[$funcidx - 2];
	my $projecttype = @{$target_ref}[$typeidx - 2];

	if (retrieve_page($elem, $hostname, $tmpfile) < 0)
	{
		return;
	}

	$prjfunc->($name, $hostname, $elem, $tmpfile, $projecttype, $category);
	
	unlink($tmpfile);
}

sub project_main_thread
{
	my $hostname = $_[0];
	my $elem;
	my $project_name;
	my $category;
	my $tmpfile;
	my $projectnum = 0;
	my @mythreads;
	my $thread;

	while(1)
	{
		mylog_f("project_main_thread($hostname): threads(%d), urls(%d)\n", $#mythreads + 1, $#project_urls + 1);
	
		while (!can_new_thread(\@mythreads, $PROJECT_THREADS_NUM))
		{
			sleep(3);
		}
		
		# get url first, then create thread
		$project_urls_sem->down();
		
		$project_urls_mutex->down();
		
		if (!$target_num && (($#project_urls + 1) <= 0))
		{
			$project_urls_mutex->up();
			last;
		}

		$elem = shift @project_urls;
		$project_name = shift @project_names;
		$category = shift @project_categorys;
		
		$project_urls_mutex->up();

		if (!$elem)
		{
			#mylog("bug, project_urls is null\n");
			next;
		}
		
		mylog("retrieve_page(".$hostname."-".$projectnum."): ".$project_name."\n");

		$tmpfile = $hostname.$projectnum.".html";

		$thread = threads->create(\&project_thread, $project_name, $hostname, $elem, $tmpfile, $category);
		push(@mythreads, $thread);
		
		unlink($tmpfile);

		$projectnum++;
	}
	
	mylog("project_main_thread: wait_child_thread begin\n");
	wait_child_thread(\@mythreads);
	mylog("project_main_thread: wait_child_thread end\n");
	
	mylog("********exit because target_num is zero********\n");
}

sub read_old_urls
{
	my $FILE;
	my $line;
	my $site = $_[0];
	
	if (!open($FILE, "<$urlfile"))
	{
		return;
	}
	
	while ($line = <$FILE>)
	{
		chomp($line);
		
		if (!($line eq "") && ($line =~ /$site/))
		{
			$oldurls{$line} = 1;
		}
	}

	close($FILE);
}

sub getTime
{
	#time() seconds return from 1970-01-01
	my $time = shift || time();
	
	my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime($time);
	
	$mon ++;
	$sec  = ($sec<10)?"0$sec":$sec;
	$min  = ($min<10)?"0$min":$min;
	$hour = ($hour<10)?"0$hour":$hour;
	$mday = ($mday<10)?"0$mday":$mday;
	$mon  = ($mon<9)?"0".$mon:$mon;
	$year+=1900;

	my $weekday = ('Sun','Mon','Tue','Wed','Thu','Fri','Sat')[$wday];
	return {
		'second' => $sec,
		'minute' => $min,
		'hour'   => $hour,
		'day'    => $mday,
		'month'  => $mon,
		'year'   => $year,
		'weekNo' => $wday,
		'wday'   => $weekday,
		'yday'   => $yday,
		'date'   => "$year$mon$mday"
		};
}

sub init_file
{
	my $timestamp=time;  
	my %mytime = %{getTime()};
	my $year = $mytime{'year'};
	my $mon = $mytime{'month'};
	my $mday = $mytime{'day'};
	
	$resultfile = "products-$year-"."$mon-"."$mday".".xml";

	open($RESULTHND, ">$resultfile") || die("open $resultfile failed\n");
	
	print $RESULTHND "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n";
	print $RESULTHND "<ExternalData>\n";
  print $RESULTHND "<Products>\n";
	
	if (!(-e $urlfile))
	{
		open($URLHND, ">$urlfile") || die("open $urlfile failed\n");
	}
	else
	{
		open($URLHND, ">>$urlfile") || die("open $urlfile failed\n");	#xxx: keep old urls
	}

	open($LOGHND, ">$logfile") || die("open $logfile failed\n");

	$URLHND->autoflush(1);
	$LOGHND->autoflush(1);
}

sub init_pids
{
	if(-e $pidsdir)
	{
		my $DIR;
		
		opendir($DIR, $pidsdir) || die $!;
		
		while (my $file = readdir($DIR))
		{
			if (-f "$pidsdir/$file")
			{
				unlink "$pidsdir/$file";
			}
		}
		
		closedir $DIR;
	}
	else
	{
		mkdir($pidsdir, 0777) || die "cannot create dir"; 
	}
	
	if ($get_picture_flag)
	{
		if(-e $picdir)
		{

		}
		else
		{
			mkdir($picdir, 0777) || die "cannot create dir"; 
		}
	}
}

sub wait_father
{
	my $pid = $_[0];
	
	while (!(-e "$pidsdir"."/$pid"))
	{
		#xxx: dead loop???
		sleep(1);
	}
}

sub wait_child
{
	my $child;
	my $mychilds = $_[0];
	my $sleep_times = 0;
	
	while ($child = shift @{$mychilds})
	{
		#if (waitpid($child, WNOHANG) == $child)
		if (waitpid($child, 0) == $child)
		{
			mylog("waitpid ".$child." successfully\n");
			next;
		}
		else
		{
			if ($sleep_times >= 10)
			{
				last;
			}
			
			$sleep_times++;
			
			mylog("sleep $sleep_times times\n");
			push(@{$mychilds}, $child);
			sleep(3);
		}
	}

	mylog("kill process begin\n");

	while ($child = shift @{$mychilds})
	{
		kill(9, $child);
	}
	
	mylog("kill process end\n");
}

sub site_process
{
	my $searchkey = "void";
	my ($target_host, $target_value) = @_;
	my @target_threads;
	my $prj_main_instance;
	
	$target_num = 0;

	if ($$target_value[5] eq '0')
	{
		my @temparray;
		
		@temparray = ($target_host, $searchkey, @$target_value);
		push(@searchtargets, \@temparray);
	}
	else
	{
		for (my $i = 0; $i < $#searchkeywords + 1; $i++)
		{
			my @temparray;
	
			$searchkey = $searchkeywords[$i];
	
			#$hostname, $searchkey,    @$target_value,                               
			#(hostname, categorystr, (searchstr, linkstr, funcstr, finalstr, typestr, keyflag))
			@temparray = ($target_host, $searchkey, @$target_value);
			push(@searchtargets, \@temparray);
		}
	}

	$target_num = $#searchtargets + 1;
	
	if ($target_num > 0)
	{
		my $thread;
		my $loopnum = $target_num;
	
		for (my $i = 0; $i < $loopnum; $i++)
		{
			$next_flags_mutex->down();
			$target_next_flags[$i] = 1;
			$next_flags_mutex->up();
		
			$thread = threads->create(\&target_main_thread, $i);
			
			if (!defined($thread))
			{
				die("new target main thread failed\n");
			}
			
			push(@target_threads, $thread);
		}
		
		$prj_main_instance = threads->create(\&project_main_thread, $target_host);
		
		while ($thread = shift @target_threads)
		{
			$thread->join();
			mylog("one target thread joined\n");
		}
		
		$prj_main_instance->join();
	}
}

sub touch_pid_file
{
	my $PIDFILE;
	my $pid = $_[0];

	open($PIDFILE, ">$pidsdir"."/$pid") || die "open file failed\n";
	close($PIDFILE);
}

sub main_body
{
	my $target_host;
	my $target_value;
	my @mychilds;
	my $child;
	
	init_file();
	init_pids();
	
	while (($target_host, $target_value) = each %searchsites)
	{
		$TARGET_THREADS_NUM = $target_value->[$thrnum1 - 2];
		$PROJECT_THREADS_NUM = $target_value->[$thrnum2 - 2];
	
		defined($child=fork()) or die "fork process failured:$!\n";
	
		unless($child)
		{
			my $pid = $$;
			
			wait_father($pid);
			
			read_old_urls($target_host);
			site_process($target_host, $target_value);
	
			unlink("$pidsdir"."/$pid");
			exit(0);
		}
	
		touch_pid_file($child);
		push(@mychilds, $child);
	}
	
	wait_child(\@mychilds);
	
	print $RESULTHND "</Products>\n";
	print $RESULTHND "</ExternalData>\n";
	
	my $end_time = time();
	my $elapsed_time = $end_time - $^T;
	
	mylog_f("used %d seconds\n", $elapsed_time);
	
	close($RESULTHND);
	close($URLHND);
	close($LOGHND);
}

main_body();
unlink($logsemfile);
unlink($prjsemfile);
exit(0); #expicitly exit